Error in install.packages : Updating loaded packages
Error in install.packages : Updating loaded packages
Error in install.packages : Updating loaded packages
library(plyr)
library(dplyr)
library(ggplot2)
usage1 = read.csv("H:\\Downloads\\MOOC DATA\\usages.effec1.csv")
usage1

usage2 = read.csv("H:\\Downloads\\MOOC DATA\\usages.effec2.csv")
usage2

usage3 = read.csv("H:\\Downloads\\MOOC DATA\\usages.effec3.csv")
usage3
effecquest1 = read.csv("H:\\Downloads\\MOOC DATA\\effec1.quest.compil.csv")
effecquest1
install.packages("naniar")
WARNING: Rtools is required to build R packages but is not currently installed. Please download and install the appropriate version of Rtools before proceeding:

https://cran.rstudio.com/bin/windows/Rtools/
Warning in install.packages :
  package ‘naniar’ is in use and will not be installed
install.packages("plyr")
WARNING: Rtools is required to build R packages but is not currently installed. Please download and install the appropriate version of Rtools before proceeding:

https://cran.rstudio.com/bin/windows/Rtools/
Warning in install.packages :
  package ‘plyr’ is in use and will not be installed
install.packages("dplyr")
WARNING: Rtools is required to build R packages but is not currently installed. Please download and install the appropriate version of Rtools before proceeding:

https://cran.rstudio.com/bin/windows/Rtools/
Warning in install.packages :
  package ‘dplyr’ is in use and will not be installed
effecquest2 = read.csv("H:\\Downloads\\MOOC DATA\\effec2.quest.compil.csv", fileEncoding = "ISO-8859-1")
effecquest2

effecquest3 = read.csv("H:\\Downloads\\MOOC DATA\\effec3.quest.compil.csv")
effecquest3
effecquest1 <- select(effecquest1, "Student_ID",)
effecquest1

effecquest2 <- select(effecquest2, "Country" , "Formation" , "Diploma" , "Student_ID" , "Gender")
effecquest2

effecquest3 <- select (effecquest3, "Country" , "Formation" , "Diploma" , "Student_ID" , "Gender")
effecquest3
datamerge1 <- merge(usage1, effecquest1, by = "Student_ID" )
datamerge2 <- merge(usage2, effecquest2, by = "Student_ID" )
datamerge3 <- merge(usage3, effecquest3, by = "Student_ID" )
datamerge1
datamerge2
datamerge3
datamerge1 <- datamerge1 %>% mutate(iteration = 1)
datamerge2 <- datamerge2 %>% mutate(iteration = 2)
datamerge3 <- datamerge3 %>% mutate(iteration = 3)

finaldata <- rbind.fill(datamerge1, datamerge2, datamerge3)
finaldata
colnames(finaldata)
 [1] "Student_ID"            "Exam.score"            "Exam.bin"              "Assignment.score"      "Assignment.bin"       
 [6] "Quizz.1.score"         "Quizz.1.bin"           "Quizz.2.score"         "Quizz.2.bin"           "Quizz.3.score"        
[11] "Quizz.3.bin"           "Quizz.4.bin"           "Quizz.4.score"         "Quizz.5.bin"           "Quizz.5.score"        
[16] "Intro.MOOC"            "Prez.sem.1"            "S1.L1"                 "S1.L2"                 "S1.L3"                
[21] "S1.L4"                 "S1.L5"                 "S1.L6"                 "Prez.sem.2"            "S2.L1"                
[26] "S2.L2"                 "S2.L3"                 "S2.L4"                 "S2.L5"                 "S2.L6"                
[31] "Prez.sem.3"            "S3.L1.1"               "S3.L1.2"               "S3.L2"                 "S3.L3"                
[36] "S3.L4"                 "S3.L5"                 "Prez.sem.4"            "S4.L1.1"               "S4.L1.2"              
[41] "S4.L2"                 "S4.L3"                 "S4.L4"                 "S4.L5"                 "Prez.sem.5"           
[46] "S5.L1.1"               "S5.L1.2"               "S5.L2"                 "S5.L3"                 "S5.L4"                
[51] "S5.L5"                 "Post.forum.0"          "view.forum.0"          "Post.forum.1"          "Post.forum.1.2"       
[56] "view.forum.1"          "view.forum.1.2"        "Post.forum.2"          "Post.forum.2.2"        "view.forum.2"         
[61] "view.forum.2.2"        "Post.forum.3"          "view.forum.3"          "Post.forum.4"          "Post.forum.4.2"       
[66] "view.forum.4"          "view.forum.4.2"        "Post.forum.5"          "Post.forum.5.2"        "view.forum.5"         
[71] "view.forum.5.2"        "last.video"            "last.quizz"            "iteration"             "Assignment.choice"    
[76] "Country"               "Formation"             "Diploma"               "Gender"                "Post.forum.fonc.cours"
[81] "view.forum.fonc.cours"
library(dplyr)

finaldata <- finaldata %>%
  
  mutate(Category = ifelse(Exam.bin == 1 | Assignment.bin == 1, "Completer", NA)) %>%
  
  mutate(Category = ifelse(Exam.bin == 0 & (Assignment.bin == 1 |
                                            Quizz.1.bin == 1 |
                                            Quizz.2.bin == 1 |
                                            Quizz.3.bin == 1 |
                                            Quizz.4.bin == 1 |
                                            Quizz.5.bin == 1), 
                                            "Disengaging Learner", Category)) %>%
  
  mutate(Category = ifelse(Exam.bin == 0 & Assignment.bin == 0 &
                                            Quizz.1.bin == 0 &
                                            Quizz.2.bin == 0 &
                                            Quizz.3.bin == 0 &
                                            Quizz.4.bin == 0 &
                                            Quizz.5.bin == 0 &
                                            last.video > 10, 
                                            "Auditing Learner", Category)) %>%
  
  mutate(Category = ifelse(Exam.bin == 0 & Assignment.bin == 0 &
                                            Quizz.1.bin == 0 &
                                            Quizz.2.bin == 0 &
                                            Quizz.3.bin == 0 &
                                            Quizz.4.bin == 0 &
                                            Quizz.5.bin == 0 &
                                            last.video <= 10, 
                                            "Bystander", Category))

print(finaldata)
data_summary <- finaldata %>%
  group_by(iteration, Category) %>%
  dplyr::summarize(count = n()) %>%
  ungroup()
`summarise()` has grouped output by 'iteration'. You can override using the `.groups` argument.
#The Plot for percentage of student status by iteration
ggplot(data_summary, aes(x = factor(iteration), y = count, fill = Category)) +
  geom_bar(stat = "identity", position = "fill") +  
  labs(title = "Percentage of Student Status by Iteration",
       x = "Iteration",
       y = "Percentage") +
  theme_minimal() +
  scale_y_continuous(labels = scales::percent_format()) 

Error in install.packages : Updating loaded packages
library(vcd)

mosaic(~ iteration + Category,
       direction = c("v", "h"),
       data = finaldata,
       shade = TRUE,
       main = "Mosaic Plot of Learner Categories by Iteration",
       xlab = "Iteration",
       ylab = "Category")

This is the start of the PHD dataset to identify errors in the data:

Phd_data<-read.csv("H:\\Downloads\\PhD.dataset.csv",header=T,na.strings="")
Phd_data
summary(Phd_data)
       X             Auteur          Identifiant.auteur    Titre           Directeur.de.these
 Min.   :     0   Length:448047      Length:448047      Length:448047      Length:448047     
 1st Qu.:112012   Class :character   Class :character   Class :character   Class :character  
 Median :224023   Mode  :character   Mode  :character   Mode  :character   Mode  :character  
 Mean   :224023                                                                              
 3rd Qu.:336035                                                                              
 Max.   :448046                                                                              
                                                                                             
 Directeur.de.these..nom.prenom. Identifiant.directeur Etablissement.de.soutenance Identifiant.etablissement
 Length:448047                   Length:448047         Length:448047               Length:448047            
 Class :character                Class :character      Class :character            Class :character         
 Mode  :character                Mode  :character      Mode  :character            Mode  :character         
                                                                                                            
                                                                                                            
                                                                                                            
                                                                                                            
  Discipline           Statut          Date.de.premiere.inscription.en.doctorat Date.de.soutenance      Year      
 Length:448047      Length:448047      Length:448047                            Length:448047      Min.   :1971   
 Class :character   Class :character   Class :character                         Class :character   1st Qu.:1994   
 Mode  :character   Mode  :character   Mode  :character                         Mode  :character   Median :2004   
                                                                                                   Mean   :2003   
                                                                                                   3rd Qu.:2012   
                                                                                                   Max.   :2020   
                                                                                                   NA's   :57086  
 Langue.de.la.these Identifiant.de.la.these Accessible.en.ligne Publication.dans.theses.fr Mise.a.jour.dans.theses.fr
 Length:448047      Length:448047           Length:448047       Length:448047              Length:448047             
 Class :character   Class :character        Class :character    Class :character           Class :character          
 Mode  :character   Mode  :character        Mode  :character    Mode  :character           Mode  :character          
                                                                                                                     
                                                                                                                     
                                                                                                                     
                                                                                                                     
 Discipline_prÃ.di     Genre           etablissement_rec   Langue_rec       
 Length:448047      Length:448047      Length:448047      Length:448047     
 Class :character   Class :character   Class :character   Class :character  
 Mode  :character   Mode  :character   Mode  :character   Mode  :character  
                                                                            
                                                                            
                                                                            
                                                                            
str(Phd_data)
'data.frame':   448047 obs. of  23 variables:
 $ X                                       : int  0 1 2 3 4 5 6 7 8 9 ...
 $ Auteur                                  : chr  "Saeed Al marri" "Andrea Ramazzotti" "OLIVIER BODENREIDER" "Emmanuel Porte" ...
 $ Identifiant.auteur                      : chr  NA "174423705" NA NA ...
 $ Titre                                   : chr  "Le credit documentaire et l'onopposabilite des exceptions" "Application de la PGD a la resolution de problemes transitoires couples en vue de l'allegement des structures composites." "Conception d'un outil informatique d'etude des cinetiques observees en toxicologie clinique" "Socio-histoire des politiques publiques en matiere sociale concernant les etudiants." ...
 $ Directeur.de.these                      : chr  "Philippe Delebecque" "Jean-Claude Grandidier,Marianne Beringhier" "Francois Kohler" "Gilles Pollet" ...
 $ Directeur.de.these..nom.prenom.         : chr  "Delebecque Philippe" "Grandidier Jean-Claude,Beringhier Marianne" "Kohler Francois" "Pollet Gilles" ...
 $ Identifiant.directeur                   : chr  "29561248" "715,441,511" "57030758" "na" ...
 $ Etablissement.de.soutenance             : chr  "Paris 1" "Chasseneuil-du-Poitou, Ecole nationale superieure de mecanique et d'aerotechnique" "Nancy 1" "Lyon 2" ...
 $ Identifiant.etablissement               : chr  "27361802" "28024400" NA "02640334X" ...
 $ Discipline                              : chr  "Driot prive" "Mecanique des solides, des materiaux, des structures et des surfaces" "Medecine" "Science politique" ...
 $ Statut                                  : chr  "enCours" "enCours" "soutenue" "enCours" ...
 $ Date.de.premiere.inscription.en.doctorat: chr  "30-09-11" "01-10-12" NA "01-06-11" ...
 $ Date.de.soutenance                      : chr  NA NA "01-01-93" NA ...
 $ Year                                    : num  NA NA 1993 NA NA ...
 $ Langue.de.la.these                      : chr  "na" "na" "fr" "na" ...
 $ Identifiant.de.la.these                 : chr  "s69480" "s98826" "1993NAN19006" "s88867" ...
 $ Accessible.en.ligne                     : chr  "non" "non" "non" "non" ...
 $ Publication.dans.theses.fr              : chr  "26-01-12" "22-11-13" "24-05-13" "12-07-13" ...
 $ Mise.a.jour.dans.theses.fr              : chr  "26-01-12" "22-11-13" "17-11-12" "12-01-16" ...
 $ Discipline_prÃ.di                       : chr  "Droit et Science Politique" "Materiaux, Milieux et Chimie" "Medecine" "Droit et Science Politique" ...
 $ Genre                                   : chr  "male" "female" "male" "male" ...
 $ etablissement_rec                       : chr  "Université Paris 1 - Panthéon Sorbonne" "École nationale supérieure de mécanique et d'aérotechnique de Poitiers" "Université de Lorraine" "Université Lumière - Lyon 2" ...
 $ Langue_rec                              : chr  "NA" "NA" "Français" "NA" ...
colnames(Phd_data)
 [1] "X"                                        "Auteur"                                  
 [3] "Identifiant.auteur"                       "Titre"                                   
 [5] "Directeur.de.these"                       "Directeur.de.these..nom.prenom."         
 [7] "Identifiant.directeur"                    "Etablissement.de.soutenance"             
 [9] "Identifiant.etablissement"                "Discipline"                              
[11] "Statut"                                   "Date.de.premiere.inscription.en.doctorat"
[13] "Date.de.soutenance"                       "Year"                                    
[15] "Langue.de.la.these"                       "Identifiant.de.la.these"                 
[17] "Accessible.en.ligne"                      "Publication.dans.theses.fr"              
[19] "Mise.a.jour.dans.theses.fr"               "Discipline_prÃ.di"                       
[21] "Genre"                                    "etablissement_rec"                       
[23] "Langue_rec"                              
library(naniar)
library(tidyverse)
library(dplyr)
percentage_miss <- sapply(Phd_data, function(x) sum(is.na(x)) / length(x) * 100)
percentage_miss
                                       X                                   Auteur                       Identifiant.auteur 
                            0.000000e+00                             0.000000e+00                             2.909226e+01 
                                   Titre                       Directeur.de.these          Directeur.de.these..nom.prenom. 
                            1.562336e-03                             2.901481e-03                             2.901481e-03 
                   Identifiant.directeur              Etablissement.de.soutenance                Identifiant.etablissement 
                            0.000000e+00                             2.231909e-04                             3.812546e+00 
                              Discipline                                   Statut Date.de.premiere.inscription.en.doctorat 
                            0.000000e+00                             0.000000e+00                             8.564191e+01 
                      Date.de.soutenance                                     Year                       Langue.de.la.these 
                            1.274107e+01                             1.274107e+01                             0.000000e+00 
                 Identifiant.de.la.these                      Accessible.en.ligne               Publication.dans.theses.fr 
                            0.000000e+00                             0.000000e+00                             0.000000e+00 
              Mise.a.jour.dans.theses.fr                        Discipline_prÃ.di                                    Genre 
                            3.950478e-02                             0.000000e+00                             0.000000e+00 
                       etablissement_rec                               Langue_rec 
                            6.860887e-01                             0.000000e+00 
vis_miss(Phd_data, warn_large_data = FALSE)
install.packages("vcd")
WARNING: Rtools is required to build R packages but is not currently installed. Please download and install the appropriate version of Rtools before proceeding:

https://cran.rstudio.com/bin/windows/Rtools/
Warning in install.packages :
  package ‘vcd’ is in use and will not be installed

Phd_data_binary <- Phd_data %>%
  mutate(across(everything(), ~ ifelse(is.na(.), 0, 1)))
cor_matrix <- cor(Phd_data_binary)
Warning in cor(Phd_data_binary) : the standard deviation is zero
print(cor_matrix)
                                          X Auteur Identifiant.auteur         Titre Directeur.de.these
X                                         1     NA                 NA            NA                 NA
Auteur                                   NA      1                 NA            NA                 NA
Identifiant.auteur                       NA     NA        1.000000000  6.170903e-03       3.848084e-03
Titre                                    NA     NA        0.006170903  1.000000e+00       1.048095e-01
Directeur.de.these                       NA     NA        0.003848084  1.048095e-01       1.000000e+00
Directeur.de.these..nom.prenom.          NA     NA        0.003848084  1.048095e-01       1.000000e+00
Identifiant.directeur                    NA     NA                 NA            NA                 NA
Etablissement.de.soutenance              NA     NA       -0.000956931 -5.905128e-06      -8.047387e-06
Identifiant.etablissement                NA     NA        0.198268925 -7.869352e-04       3.255071e-03
Discipline                               NA     NA                 NA            NA                 NA
Statut                                   NA     NA                 NA            NA                 NA
Date.de.premiere.inscription.en.doctorat NA     NA       -0.637333183 -8.043235e-03      -1.576861e-04
Date.de.soutenance                       NA     NA        0.593261540  8.650598e-03      -8.156306e-04
Year                                     NA     NA        0.593261540  8.650598e-03      -8.156306e-04
Langue.de.la.these                       NA     NA                 NA            NA                 NA
Identifiant.de.la.these                  NA     NA                 NA            NA                 NA
Accessible.en.ligne                      NA     NA                 NA            NA                 NA
Publication.dans.theses.fr               NA     NA                 NA            NA                 NA
Mise.a.jour.dans.theses.fr               NA     NA       -0.012733640 -7.857805e-05      -1.070846e-04
Discipline_prÃ.di                        NA     NA                 NA            NA                 NA
Genre                                    NA     NA                 NA            NA                 NA
etablissement_rec                        NA     NA        0.020580219 -3.285305e-04      -4.477146e-04
Langue_rec                               NA     NA                 NA            NA                 NA
                                         Directeur.de.these..nom.prenom. Identifiant.directeur Etablissement.de.soutenance
X                                                                     NA                    NA                          NA
Auteur                                                                NA                    NA                          NA
Identifiant.auteur                                          3.848084e-03                    NA               -9.569310e-04
Titre                                                       1.048095e-01                    NA               -5.905128e-06
Directeur.de.these                                          1.000000e+00                    NA               -8.047387e-06
Directeur.de.these..nom.prenom.                             1.000000e+00                    NA               -8.047387e-06
Identifiant.directeur                                                 NA                     1                          NA
Etablissement.de.soutenance                                -8.047387e-06                    NA                1.000000e+00
Identifiant.etablissement                                   3.255071e-03                    NA               -2.974316e-04
Discipline                                                            NA                    NA                          NA
Statut                                                                NA                    NA                          NA
Date.de.premiere.inscription.en.doctorat                   -1.576861e-04                    NA                6.117076e-04
Date.de.soutenance                                         -8.156306e-04                    NA               -5.708694e-04
Year                                                       -8.156306e-04                    NA               -5.708694e-04
Langue.de.la.these                                                    NA                    NA                          NA
Identifiant.de.la.these                                               NA                    NA                          NA
Accessible.en.ligne                                                   NA                    NA                          NA
Publication.dans.theses.fr                                            NA                    NA                          NA
Mise.a.jour.dans.theses.fr                                 -1.070846e-04                    NA               -2.969951e-05
Discipline_prÃ.di                                                     NA                    NA                          NA
Genre                                                                 NA                    NA                          NA
etablissement_rec                                          -4.477146e-04                    NA               -1.241720e-04
Langue_rec                                                            NA                    NA                          NA
                                         Identifiant.etablissement Discipline Statut
X                                                               NA         NA     NA
Auteur                                                          NA         NA     NA
Identifiant.auteur                                    0.1982689250         NA     NA
Titre                                                -0.0007869352         NA     NA
Directeur.de.these                                    0.0032550706         NA     NA
Directeur.de.these..nom.prenom.                       0.0032550706         NA     NA
Identifiant.directeur                                           NA         NA     NA
Etablissement.de.soutenance                          -0.0002974316         NA     NA
Identifiant.etablissement                             1.0000000000         NA     NA
Discipline                                                      NA          1     NA
Statut                                                          NA         NA      1
Date.de.premiere.inscription.en.doctorat              0.0814847706         NA     NA
Date.de.soutenance                                   -0.0759010138         NA     NA
Year                                                 -0.0759010138         NA     NA
Langue.de.la.these                                              NA         NA     NA
Identifiant.de.la.these                                         NA         NA     NA
Accessible.en.ligne                                             NA         NA     NA
Publication.dans.theses.fr                                      NA         NA     NA
Mise.a.jour.dans.theses.fr                           -0.0039578470         NA     NA
Discipline_prÃ.di                                               NA         NA     NA
Genre                                                           NA         NA     NA
etablissement_rec                                    -0.0135824790         NA     NA
Langue_rec                                                      NA         NA     NA
                                         Date.de.premiere.inscription.en.doctorat Date.de.soutenance          Year
X                                                                              NA                 NA            NA
Auteur                                                                         NA                 NA            NA
Identifiant.auteur                                                  -0.6373331827       0.5932615405  0.5932615405
Titre                                                               -0.0080432352       0.0086505981  0.0086505981
Directeur.de.these                                                  -0.0001576861      -0.0008156306 -0.0008156306
Directeur.de.these..nom.prenom.                                     -0.0001576861      -0.0008156306 -0.0008156306
Identifiant.directeur                                                          NA                 NA            NA
Etablissement.de.soutenance                                          0.0006117076      -0.0005708694 -0.0005708694
Identifiant.etablissement                                            0.0814847706      -0.0759010138 -0.0759010138
Discipline                                                                     NA                 NA            NA
Statut                                                                         NA                 NA            NA
Date.de.premiere.inscription.en.doctorat                             1.0000000000      -0.9286577307 -0.9286577307
Date.de.soutenance                                                  -0.9286577307       1.0000000000  1.0000000000
Year                                                                -0.9286577307       1.0000000000  1.0000000000
Langue.de.la.these                                                             NA                 NA            NA
Identifiant.de.la.these                                                        NA                 NA            NA
Accessible.en.ligne                                                            NA                 NA            NA
Publication.dans.theses.fr                                                     NA                 NA            NA
Mise.a.jour.dans.theses.fr                                           0.0081398395      -0.0075964159 -0.0075964159
Discipline_prÃ.di                                                              NA                 NA            NA
Genre                                                                          NA                 NA            NA
etablissement_rec                                                    0.0084329391      -0.0050001283 -0.0050001283
Langue_rec                                                                     NA                 NA            NA
                                         Langue.de.la.these Identifiant.de.la.these Accessible.en.ligne
X                                                        NA                      NA                  NA
Auteur                                                   NA                      NA                  NA
Identifiant.auteur                                       NA                      NA                  NA
Titre                                                    NA                      NA                  NA
Directeur.de.these                                       NA                      NA                  NA
Directeur.de.these..nom.prenom.                          NA                      NA                  NA
Identifiant.directeur                                    NA                      NA                  NA
Etablissement.de.soutenance                              NA                      NA                  NA
Identifiant.etablissement                                NA                      NA                  NA
Discipline                                               NA                      NA                  NA
Statut                                                   NA                      NA                  NA
Date.de.premiere.inscription.en.doctorat                 NA                      NA                  NA
Date.de.soutenance                                       NA                      NA                  NA
Year                                                     NA                      NA                  NA
Langue.de.la.these                                        1                      NA                  NA
Identifiant.de.la.these                                  NA                       1                  NA
Accessible.en.ligne                                      NA                      NA                   1
Publication.dans.theses.fr                               NA                      NA                  NA
Mise.a.jour.dans.theses.fr                               NA                      NA                  NA
Discipline_prÃ.di                                        NA                      NA                  NA
Genre                                                    NA                      NA                  NA
etablissement_rec                                        NA                      NA                  NA
Langue_rec                                               NA                      NA                  NA
                                         Publication.dans.theses.fr Mise.a.jour.dans.theses.fr Discipline_prÃ.di Genre
X                                                                NA                         NA                NA    NA
Auteur                                                           NA                         NA                NA    NA
Identifiant.auteur                                               NA              -1.273364e-02                NA    NA
Titre                                                            NA              -7.857805e-05                NA    NA
Directeur.de.these                                               NA              -1.070846e-04                NA    NA
Directeur.de.these..nom.prenom.                                  NA              -1.070846e-04                NA    NA
Identifiant.directeur                                            NA                         NA                NA    NA
Etablissement.de.soutenance                                      NA              -2.969951e-05                NA    NA
Identifiant.etablissement                                        NA              -3.957847e-03                NA    NA
Discipline                                                       NA                         NA                NA    NA
Statut                                                           NA                         NA                NA    NA
Date.de.premiere.inscription.en.doctorat                         NA               8.139840e-03                NA    NA
Date.de.soutenance                                               NA              -7.596416e-03                NA    NA
Year                                                             NA              -7.596416e-03                NA    NA
Langue.de.la.these                                               NA                         NA                NA    NA
Identifiant.de.la.these                                          NA                         NA                NA    NA
Accessible.en.ligne                                              NA                         NA                NA    NA
Publication.dans.theses.fr                                        1                         NA                NA    NA
Mise.a.jour.dans.theses.fr                                       NA               1.000000e+00                NA    NA
Discipline_prÃ.di                                                NA                         NA                 1    NA
Genre                                                            NA                         NA                NA     1
etablissement_rec                                                NA              -1.652326e-03                NA    NA
Langue_rec                                                       NA                         NA                NA    NA
                                         etablissement_rec Langue_rec
X                                                       NA         NA
Auteur                                                  NA         NA
Identifiant.auteur                            0.0205802186         NA
Titre                                        -0.0003285305         NA
Directeur.de.these                           -0.0004477146         NA
Directeur.de.these..nom.prenom.              -0.0004477146         NA
Identifiant.directeur                                   NA         NA
Etablissement.de.soutenance                  -0.0001241720         NA
Identifiant.etablissement                    -0.0135824790         NA
Discipline                                              NA         NA
Statut                                                  NA         NA
Date.de.premiere.inscription.en.doctorat      0.0084329391         NA
Date.de.soutenance                           -0.0050001283         NA
Year                                         -0.0050001283         NA
Langue.de.la.these                                      NA         NA
Identifiant.de.la.these                                 NA         NA
Accessible.en.ligne                                     NA         NA
Publication.dans.theses.fr                              NA         NA
Mise.a.jour.dans.theses.fr                   -0.0016523260         NA
Discipline_prÃ.di                                       NA         NA
Genre                                                   NA         NA
etablissement_rec                             1.0000000000         NA
Langue_rec                                              NA          1
library(corrgram)
gg_miss_upset(Phd_data)


missing_data <- Phd_data %>%
  is.na() %>%
  as.data.frame()

library(corrgram)

corrgram(Phd_data,
         order = TRUE,        
         lower.panel = panel.shade,   
         upper.panel = panel.pie,     
         text.panel = panel.txt,      
         main = "Corrgram of Missing Value Indicators")

library(corrplot)
library(dplyr)
na_indicator <- Phd_data %>%
  mutate(across(everything(), ~ ifelse(is.na(.), 1, 0)))

na_cor_matrix <- cor(na_indicator)
Warning in cor(na_indicator) : the standard deviation is zero
corrplot(na_cor_matrix, 
         method = "color",        
         is.corr = FALSE,         
         type = "lower",         
         tl.col = "black",    
         tl.srt = 45, #note: rotated text 45 deg so i can read the text better
         tl.cex = 0.6, #note: to make text smaller
         addCoef.col = NULL,
         col = colorRampPalette(c("blue", "white", "red"))(200),
         na.label = " ", #note: just adding this here to remove NA (all the ?)
         title = "Heatmap of Missing Value Indicators",
         mar = c(0, 0, 2, 0)) 

gg_miss_upset(Phd_data, 
              nsets = 10,           
              nintersects = 10)    

#Checking for any missing values percentage by column
sapply(Phd_data, function(x) sum(is.na(x)) / length(x) * 100)
                                       X                                   Auteur                       Identifiant.auteur 
                            0.000000e+00                             0.000000e+00                             2.909226e+01 
                                   Titre                       Directeur.de.these          Directeur.de.these..nom.prenom. 
                            1.562336e-03                             2.901481e-03                             2.901481e-03 
                   Identifiant.directeur              Etablissement.de.soutenance                Identifiant.etablissement 
                            0.000000e+00                             2.231909e-04                             3.812546e+00 
                              Discipline                                   Statut Date.de.premiere.inscription.en.doctorat 
                            0.000000e+00                             0.000000e+00                             8.564191e+01 
                      Date.de.soutenance                                     Year                       Langue.de.la.these 
                            1.274107e+01                             1.274107e+01                             0.000000e+00 
                 Identifiant.de.la.these                      Accessible.en.ligne               Publication.dans.theses.fr 
                            0.000000e+00                             0.000000e+00                             0.000000e+00 
              Mise.a.jour.dans.theses.fr                        Discipline_prÃ.di                                    Genre 
                            3.950478e-02                             0.000000e+00                             0.000000e+00 
                       etablissement_rec                               Langue_rec 
                            6.860887e-01                             0.000000e+00 
ggplot(Phd_data, aes(x = Year)) + 
  geom_bar() + 
  theme_minimal() + 
  labs(title = "The num of PHD Defenses by Year", x = "Year", y = "Count")
Warning: Removed 57086 rows containing non-finite outside the scale range (`stat_count()`).

#making sure no data is past 2020
Phd_data %>% filter(Year > 2020)
LS0tDQp0aXRsZTogIkRhdGEgV3JhbmdsaW5nIC0gUHJlcHJvY2Vzc2luZyAtIFBIRCAmIE1vb2MgRGF0YXNldCAtIEFsZXhhbmRyZSBIdWV0Ig0Kb3V0cHV0OiBodG1sX25vdGVib29rDQotLS0NCmBgYHtyIGVjaG89RkFMU0V9DQppbnN0YWxsLnBhY2thZ2VzKCJwbHlyIikNCmluc3RhbGwucGFja2FnZXMoImRwbHlyIikNCmluc3RhbGwucGFja2FnZXMoIm5hbmlhciIpDQpgYGANCmBgYHtyfQ0KbGlicmFyeShwbHlyKQ0KbGlicmFyeShkcGx5cikNCmxpYnJhcnkoZ2dwbG90MikNCmBgYA0KDQoNCmBgYHtyfQ0KdXNhZ2UxID0gcmVhZC5jc3YoIkg6XFxEb3dubG9hZHNcXE1PT0MgREFUQVxcdXNhZ2VzLmVmZmVjMS5jc3YiKQ0KdXNhZ2UxDQoNCnVzYWdlMiA9IHJlYWQuY3N2KCJIOlxcRG93bmxvYWRzXFxNT09DIERBVEFcXHVzYWdlcy5lZmZlYzIuY3N2IikNCnVzYWdlMg0KDQp1c2FnZTMgPSByZWFkLmNzdigiSDpcXERvd25sb2Fkc1xcTU9PQyBEQVRBXFx1c2FnZXMuZWZmZWMzLmNzdiIpDQp1c2FnZTMNCmBgYA0KDQoNCmBgYHtyfQ0KZWZmZWNxdWVzdDEgPSByZWFkLmNzdigiSDpcXERvd25sb2Fkc1xcTU9PQyBEQVRBXFxlZmZlYzEucXVlc3QuY29tcGlsLmNzdiIpDQplZmZlY3F1ZXN0MQ0KDQplZmZlY3F1ZXN0MiA9IHJlYWQuY3N2KCJIOlxcRG93bmxvYWRzXFxNT09DIERBVEFcXGVmZmVjMi5xdWVzdC5jb21waWwuY3N2IiwgZmlsZUVuY29kaW5nID0gIklTTy04ODU5LTEiKQ0KZWZmZWNxdWVzdDINCg0KZWZmZWNxdWVzdDMgPSByZWFkLmNzdigiSDpcXERvd25sb2Fkc1xcTU9PQyBEQVRBXFxlZmZlYzMucXVlc3QuY29tcGlsLmNzdiIpDQplZmZlY3F1ZXN0Mw0KYGBgDQpgYGB7cn0NCmVmZmVjcXVlc3QxIDwtIHNlbGVjdChlZmZlY3F1ZXN0MSwgIlN0dWRlbnRfSUQiLCkNCmVmZmVjcXVlc3QxDQoNCmVmZmVjcXVlc3QyIDwtIHNlbGVjdChlZmZlY3F1ZXN0MiwgIkNvdW50cnkiICwgIkZvcm1hdGlvbiIgLCAiRGlwbG9tYSIgLCAiU3R1ZGVudF9JRCIgLCAiR2VuZGVyIikNCmVmZmVjcXVlc3QyDQoNCmVmZmVjcXVlc3QzIDwtIHNlbGVjdCAoZWZmZWNxdWVzdDMsICJDb3VudHJ5IiAsICJGb3JtYXRpb24iICwgIkRpcGxvbWEiICwgIlN0dWRlbnRfSUQiICwgIkdlbmRlciIpDQplZmZlY3F1ZXN0Mw0KYGBgDQoNCg0KYGBge3J9DQpkYXRhbWVyZ2UxIDwtIG1lcmdlKHVzYWdlMSwgZWZmZWNxdWVzdDEsIGJ5ID0gIlN0dWRlbnRfSUQiICkNCmRhdGFtZXJnZTIgPC0gbWVyZ2UodXNhZ2UyLCBlZmZlY3F1ZXN0MiwgYnkgPSAiU3R1ZGVudF9JRCIgKQ0KZGF0YW1lcmdlMyA8LSBtZXJnZSh1c2FnZTMsIGVmZmVjcXVlc3QzLCBieSA9ICJTdHVkZW50X0lEIiApDQpkYXRhbWVyZ2UxDQpkYXRhbWVyZ2UyDQpkYXRhbWVyZ2UzDQpgYGANCg0KDQpgYGB7cn0NCmRhdGFtZXJnZTEgPC0gZGF0YW1lcmdlMSAlPiUgbXV0YXRlKGl0ZXJhdGlvbiA9IDEpDQpkYXRhbWVyZ2UyIDwtIGRhdGFtZXJnZTIgJT4lIG11dGF0ZShpdGVyYXRpb24gPSAyKQ0KZGF0YW1lcmdlMyA8LSBkYXRhbWVyZ2UzICU+JSBtdXRhdGUoaXRlcmF0aW9uID0gMykNCmBgYA0KDQpgYGB7cn0NCg0KZmluYWxkYXRhIDwtIHJiaW5kLmZpbGwoZGF0YW1lcmdlMSwgZGF0YW1lcmdlMiwgZGF0YW1lcmdlMykNCmZpbmFsZGF0YQ0KYGBgDQoNCmBgYHtyfQ0KY29sbmFtZXMoZmluYWxkYXRhKQ0KYGBgDQpgYGB7cn0NCmxpYnJhcnkoZHBseXIpDQoNCmZpbmFsZGF0YSA8LSBmaW5hbGRhdGEgJT4lDQogIA0KICBtdXRhdGUoQ2F0ZWdvcnkgPSBpZmVsc2UoRXhhbS5iaW4gPT0gMSB8IEFzc2lnbm1lbnQuYmluID09IDEsICJDb21wbGV0ZXIiLCBOQSkpICU+JQ0KICANCiAgbXV0YXRlKENhdGVnb3J5ID0gaWZlbHNlKEV4YW0uYmluID09IDAgJiAoQXNzaWdubWVudC5iaW4gPT0gMSB8DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFF1aXp6LjEuYmluID09IDEgfA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBRdWl6ei4yLmJpbiA9PSAxIHwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgUXVpenouMy5iaW4gPT0gMSB8DQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFF1aXp6LjQuYmluID09IDEgfA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBRdWl6ei41LmJpbiA9PSAxKSwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJEaXNlbmdhZ2luZyBMZWFybmVyIiwgQ2F0ZWdvcnkpKSAlPiUNCiAgDQogIG11dGF0ZShDYXRlZ29yeSA9IGlmZWxzZShFeGFtLmJpbiA9PSAwICYgQXNzaWdubWVudC5iaW4gPT0gMCAmDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFF1aXp6LjEuYmluID09IDAgJg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBRdWl6ei4yLmJpbiA9PSAwICYNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgUXVpenouMy5iaW4gPT0gMCAmDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFF1aXp6LjQuYmluID09IDAgJg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBRdWl6ei41LmJpbiA9PSAwICYNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFzdC52aWRlbyA+IDEwLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIkF1ZGl0aW5nIExlYXJuZXIiLCBDYXRlZ29yeSkpICU+JQ0KICANCiAgbXV0YXRlKENhdGVnb3J5ID0gaWZlbHNlKEV4YW0uYmluID09IDAgJiBBc3NpZ25tZW50LmJpbiA9PSAwICYNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgUXVpenouMS5iaW4gPT0gMCAmDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFF1aXp6LjIuYmluID09IDAgJg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBRdWl6ei4zLmJpbiA9PSAwICYNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgUXVpenouNC5iaW4gPT0gMCAmDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFF1aXp6LjUuYmluID09IDAgJg0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYXN0LnZpZGVvIDw9IDEwLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIkJ5c3RhbmRlciIsIENhdGVnb3J5KSkNCg0KcHJpbnQoZmluYWxkYXRhKQ0KYGBgDQpgYGB7cn0NCmRhdGFfc3VtbWFyeSA8LSBmaW5hbGRhdGEgJT4lDQogIGdyb3VwX2J5KGl0ZXJhdGlvbiwgQ2F0ZWdvcnkpICU+JQ0KICBkcGx5cjo6c3VtbWFyaXplKGNvdW50ID0gbigpKSAlPiUNCiAgdW5ncm91cCgpDQoNCiNUaGUgUGxvdCBmb3IgcGVyY2VudGFnZSBvZiBzdHVkZW50IHN0YXR1cyBieSBpdGVyYXRpb24NCmdncGxvdChkYXRhX3N1bW1hcnksIGFlcyh4ID0gZmFjdG9yKGl0ZXJhdGlvbiksIHkgPSBjb3VudCwgZmlsbCA9IENhdGVnb3J5KSkgKw0KICBnZW9tX2JhcihzdGF0ID0gImlkZW50aXR5IiwgcG9zaXRpb24gPSAiZmlsbCIpICsgIA0KICBsYWJzKHRpdGxlID0gIlBlcmNlbnRhZ2Ugb2YgU3R1ZGVudCBTdGF0dXMgYnkgSXRlcmF0aW9uIiwNCiAgICAgICB4ID0gIkl0ZXJhdGlvbiIsDQogICAgICAgeSA9ICJQZXJjZW50YWdlIikgKw0KICB0aGVtZV9taW5pbWFsKCkgKw0KICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpwZXJjZW50X2Zvcm1hdCgpKSANCmBgYA0KYGBge3IgZWNobz1GQUxTRX0NCmluc3RhbGwucGFja2FnZXMoInZjZCIpDQpgYGANCg0KYGBge3J9DQpsaWJyYXJ5KHZjZCkNCmBgYA0KDQpgYGB7cn0NCg0KbW9zYWljKH4gaXRlcmF0aW9uICsgQ2F0ZWdvcnksDQogICAgICAgZGlyZWN0aW9uID0gYygidiIsICJoIiksDQogICAgICAgZGF0YSA9IGZpbmFsZGF0YSwNCiAgICAgICBzaGFkZSA9IFRSVUUsDQogICAgICAgbWFpbiA9ICJNb3NhaWMgUGxvdCBvZiBMZWFybmVyIENhdGVnb3JpZXMgYnkgSXRlcmF0aW9uIiwNCiAgICAgICB4bGFiID0gIkl0ZXJhdGlvbiIsDQogICAgICAgeWxhYiA9ICJDYXRlZ29yeSIpDQpgYGANClRoaXMgaXMgdGhlIHN0YXJ0IG9mIHRoZSBQSEQgZGF0YXNldCB0byBpZGVudGlmeSBlcnJvcnMgaW4gdGhlIGRhdGE6DQoNCmBgYHtyfQ0KUGhkX2RhdGE8LXJlYWQuY3N2KCJIOlxcRG93bmxvYWRzXFxQaEQuZGF0YXNldC5jc3YiLGhlYWRlcj1ULG5hLnN0cmluZ3M9IiIpDQpQaGRfZGF0YQ0KYGBgDQoNCmBgYHtyfQ0Kc3VtbWFyeShQaGRfZGF0YSkNCnN0cihQaGRfZGF0YSkNCmNvbG5hbWVzKFBoZF9kYXRhKQ0KYGBgDQpgYGB7cn0NCmxpYnJhcnkobmFuaWFyKQ0KbGlicmFyeSh0aWR5dmVyc2UpDQpsaWJyYXJ5KGRwbHlyKQ0KYGBgDQoNCmBgYHtyfQ0KcGVyY2VudGFnZV9taXNzIDwtIHNhcHBseShQaGRfZGF0YSwgZnVuY3Rpb24oeCkgc3VtKGlzLm5hKHgpKSAvIGxlbmd0aCh4KSAqIDEwMCkNCnBlcmNlbnRhZ2VfbWlzcw0KYGBgDQoNCmBgYHtyfQ0KdmlzX21pc3MoUGhkX2RhdGEsIHdhcm5fbGFyZ2VfZGF0YSA9IEZBTFNFKQ0KYGBgDQoNCmBgYHtyfQ0KUGhkX2RhdGFfYmluYXJ5IDwtIFBoZF9kYXRhICU+JQ0KICBtdXRhdGUoYWNyb3NzKGV2ZXJ5dGhpbmcoKSwgfiBpZmVsc2UoaXMubmEoLiksIDAsIDEpKSkNCmNvcl9tYXRyaXggPC0gY29yKFBoZF9kYXRhX2JpbmFyeSkNCg0KcHJpbnQoY29yX21hdHJpeCkNCmBgYA0KYGBge3J9DQpsaWJyYXJ5KGNvcnJncmFtKQ0KYGBgDQoNCmBgYHtyfQ0KZ2dfbWlzc191cHNldChQaGRfZGF0YSkNCg0KbWlzc2luZ19kYXRhIDwtIFBoZF9kYXRhICU+JQ0KICBpcy5uYSgpICU+JQ0KICBhcy5kYXRhLmZyYW1lKCkNCg0KbGlicmFyeShjb3JyZ3JhbSkNCg0KY29ycmdyYW0oUGhkX2RhdGEsDQogICAgICAgICBvcmRlciA9IFRSVUUsICAgICAgICANCiAgICAgICAgIGxvd2VyLnBhbmVsID0gcGFuZWwuc2hhZGUsICAgDQogICAgICAgICB1cHBlci5wYW5lbCA9IHBhbmVsLnBpZSwgICAgIA0KICAgICAgICAgdGV4dC5wYW5lbCA9IHBhbmVsLnR4dCwgICAgICANCiAgICAgICAgIG1haW4gPSAiQ29ycmdyYW0gb2YgTWlzc2luZyBWYWx1ZSBJbmRpY2F0b3JzIikNCmBgYA0KDQpgYGB7cn0NCmxpYnJhcnkoY29ycnBsb3QpDQpsaWJyYXJ5KGRwbHlyKQ0KYGBgDQoNCmBgYHtyfQ0KbmFfaW5kaWNhdG9yIDwtIFBoZF9kYXRhICU+JQ0KICBtdXRhdGUoYWNyb3NzKGV2ZXJ5dGhpbmcoKSwgfiBpZmVsc2UoaXMubmEoLiksIDEsIDApKSkNCg0KbmFfY29yX21hdHJpeCA8LSBjb3IobmFfaW5kaWNhdG9yKQ0KDQpjb3JycGxvdChuYV9jb3JfbWF0cml4LCANCiAgICAgICAgIG1ldGhvZCA9ICJjb2xvciIsICAgICAgICANCiAgICAgICAgIGlzLmNvcnIgPSBGQUxTRSwgICAgICAgICANCiAgICAgICAgIHR5cGUgPSAibG93ZXIiLCAgICAgICAgIA0KICAgICAgICAgdGwuY29sID0gImJsYWNrIiwgICAgDQogICAgICAgICB0bC5zcnQgPSA0NSwgI25vdGU6IHJvdGF0ZWQgdGV4dCA0NSBkZWcgc28gaSBjYW4gcmVhZCB0aGUgdGV4dCBiZXR0ZXINCiAgICAgICAgIHRsLmNleCA9IDAuNiwgI25vdGU6IHRvIG1ha2UgdGV4dCBzbWFsbGVyDQogICAgICAgICBhZGRDb2VmLmNvbCA9IE5VTEwsDQogICAgICAgICBjb2wgPSBjb2xvclJhbXBQYWxldHRlKGMoImJsdWUiLCAid2hpdGUiLCAicmVkIikpKDIwMCksDQogICAgICAgICBuYS5sYWJlbCA9ICIgIiwgI25vdGU6IGp1c3QgYWRkaW5nIHRoaXMgaGVyZSB0byByZW1vdmUgTkEgKGFsbCB0aGUgPykNCiAgICAgICAgIHRpdGxlID0gIkhlYXRtYXAgb2YgTWlzc2luZyBWYWx1ZSBJbmRpY2F0b3JzIiwNCiAgICAgICAgIG1hciA9IGMoMCwgMCwgMiwgMCkpIA0KDQpgYGANCmBgYHtyfQ0KZ2dfbWlzc191cHNldChQaGRfZGF0YSwgDQogICAgICAgICAgICAgIG5zZXRzID0gMTAsICAgICAgICAgICANCiAgICAgICAgICAgICAgbmludGVyc2VjdHMgPSAxMCkgICAgDQpgYGANCmBgYHtyfQ0KI0NoZWNraW5nIGZvciBhbnkgbWlzc2luZyB2YWx1ZXMgcGVyY2VudGFnZSBieSBjb2x1bW4NCnNhcHBseShQaGRfZGF0YSwgZnVuY3Rpb24oeCkgc3VtKGlzLm5hKHgpKSAvIGxlbmd0aCh4KSAqIDEwMCkNCmdncGxvdChQaGRfZGF0YSwgYWVzKHggPSBZZWFyKSkgKyANCiAgZ2VvbV9iYXIoKSArIA0KICB0aGVtZV9taW5pbWFsKCkgKyANCiAgbGFicyh0aXRsZSA9ICJUaGUgbnVtIG9mIFBIRCBEZWZlbnNlcyBieSBZZWFyIiwgeCA9ICJZZWFyIiwgeSA9ICJDb3VudCIpDQpgYGANCg0KYGBge3J9DQojbWFraW5nIHN1cmUgbm8gZGF0YSBpcyBwYXN0IDIwMjANClBoZF9kYXRhICU+JSBmaWx0ZXIoWWVhciA+IDIwMjApDQpgYGA=